# coding=utf8
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, ElementNotVisibleException
from configuration import DELAY, SLEEP, WAIT, LOG_FILE, TIMEOUT
import time
import re
from utilities import convert_str_to_datetime
from objects import ExceptionalEnterprise, ExceptionalReason
import database_objects
from utilities import Timer
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.common.exceptions import NoAlertPresentException, UnexpectedAlertPresentException
from selenium.webdriver.common.alert import Alert
# from pyvirtualdisplay import Display


class EESpider(object):
    '''
    Exceptional Enterprise Spider
    '''

    def __init__(self, start_page=1):
        # options = webdriver.ChromeOptions()
        # options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"])
        # self.browser = webdriver.Chrome(chrome_options=options)
        # self.browser = webdriver.Ie()
        binary = FirefoxBinary(r'C:/Program File (x86)/Mozilla Firefox/firefox.exe')
        # self.display = Display(visible=0, size=(1024, 768))
        # self.display.start()
        self.browser = webdriver.Firefox()
        self.main_url = 'http://www.jsgsj.gov.cn:58888/province/notice/QueryExceptionDirectory.jsp'
        self.start_page = start_page
        self.current_page = 1

        self.latest_time = database_objects.ExceptionalEnterprise.get_latest_time()

        self.ee_list = []  # exceptional enterprise list
        self.reason_list = []

    def get_page_ready(self):
        self.browser.get(self.main_url)
        self.wait()  # wait for page ready

        if self.start_page == 1:
            return 0

        while self.start_page != self.current_page:
            pages = self.browser.find_elements_by_class_name('clickPage1')
            last_page = int(pages[-1].text)
            if self.start_page > last_page:
                self.browser.find_element_by_xpath('//*[@id="fenye"]/ul/li[11]/a').send_keys('\n')
                timer = Timer()
                while True:
                    try:
                        pages = self.browser.find_elements_by_class_name('clickPage1')
                        current_page = pages[-1]
                        style = current_page.get_attribute('style')
                        if style != u'':
                            self.current_page = last_page
                            print 'Page %d loaded!' % (self.current_page,)
                            break
                        else:
                            if timer.loading_timeout() is True:
                                self.browser.refresh()
                                print 'Refresh page...'
                    except Exception as e:
                        continue
                self.go_next_page()
            else:
                diff = last_page - self.start_page
                self.browser.find_element_by_xpath('//*[@id="fenye"]/ul/li[%d]/a' % (11 - diff,)).send_keys('\n')
                timer = Timer()
                while True:
                    try:
                        pages = self.browser.find_elements_by_class_name('clickPage1')
                        current_page = pages[9 - diff]
                        style = current_page.get_attribute('style')
                        if style != u'':
                            self.current_page = last_page - diff
                            print 'Page %d loaded!' % (self.current_page,)
                            break
                        else:
                            if timer.loading_timeout() is True:
                                self.browser.refresh()
                                print 'Refresh page...'
                    except Exception as e:
                        continue

    def wait_for_click(self):
        while True:
            try:
                print '1111111'
                self.browser.switch_to.window(self.browser.window_handles[1])
                break
            except IndexError as ie:
                time.sleep(SLEEP)
            except UnexpectedAlertPresentException as uape:
                self.accept_alert()

    def wait(self, element_xpath='/html'):
        timer = Timer()
        while True:
            time.sleep(DELAY)
            try:
                content = self.browser.find_element_by_xpath(element_xpath)
                break
            except NoSuchElementException as nsee:
                if timer.loading_timeout() is True:
                    self.browser.refresh()
                    print 'Refresh page...'
                continue

    def get_one_page(self):
        i = 1
        while True:
            try:
                exceptional_enterprise = self.browser.find_element_by_xpath('//*[@id="tbody_id1"]/div[%d]' % (i,))
            except NoSuchElementException as e:
                print 'Page %d global information collected!' % (self.current_page,)
                break
            i += 1
            info = exceptional_enterprise.text.split(u'\n')
            if len(info) == 3:
                name = info[0]
                register_id = info[1]
                listed_date = convert_str_to_datetime(info[2].encode('utf-8'))
                # if listed_date == self.latest_time:
                #     print 'Crawl Completed!'
                #     exit(2)
                self.ee_list.append(ExceptionalEnterprise(name, register_id, listed_date))
            else:
                print 'Discard one entry! Index: %d' % (i-1,)

    def get_register_id(self):
        data = None
        while True:
            try:
                data = self.browser.find_element_by_xpath('//*[@id="parent_id1"]/div/h2').text
                break
            except NoSuchElementException as nsee:
                self.browser.refresh()
                time.sleep(SLEEP)
            except UnexpectedAlertPresentException as uape:
                self.browser.refresh()
                self.accept_alert()

        try:
            register_id = re.split(r'\s+', data)[1]
            register_id = register_id.split(u'：')[1]
        except Exception as e:
            print e.message

        return register_id

    def get_reasons(self):
        register_id = self.get_register_id()
        index = 3
        while True:
            try:
                in_reason = self.browser.find_element_by_xpath(
                    '//*[@id="tbody_id10"]/tr[%d]/td[2]/span' % (index,)
                ).get_attribute('completedata')
            except NoSuchElementException as nsee:
                print 'Register ID: %s gets all records!' % (register_id,)
                break
            except UnexpectedAlertPresentException as nape:
                self.accept_alert()
            print 'Register ID: %s gets Record %d!' % (register_id, index-2)
            in_date = self.browser.find_element_by_xpath('//*[@id="tbody_id10"]/tr[%d]/td[3]' % (index,))
            out_reason = self.browser.find_element_by_xpath('//*[@id="tbody_id10"]/tr[%d]/td[4]' % (index,))
            out_date = self.browser.find_element_by_xpath('//*[@id="tbody_id10"]/tr[%d]/td[5]' % (index,))
            judger = self.browser.find_element_by_xpath('//*[@id="tbody_id10"]/tr[%d]/td[6]' % (index,))
            index += 1

            in_date = convert_str_to_datetime(in_date.text.encode('utf-8'))
            out_reason = out_reason.text
            out_date = out_date.text.strip(u' ')
            if out_date != u'':
                out_date = convert_str_to_datetime(out_date.encode('utf-8'))
            else:
                out_date = None
            judger = judger.text

            self.reason_list.append(ExceptionalReason(
                    register_id=register_id,
                    in_reason=in_reason,
                    in_date=in_date,
                    out_reason=out_reason,
                    out_date=out_date,
                    judger=judger
                )
            )

    def close_current_tab(self):
        self.browser.close()
        self.browser.switch_to_window(self.browser.window_handles[0])

    def go_next_page(self):
        if self.current_page > self.start_page:
            with open(LOG_FILE, 'w') as writer:
                writer.write(str(self.current_page))

        self.current_page += 1
        self.browser.find_element_by_xpath('//*[@id="nextPage1"]').send_keys('\n')
        time.sleep(DELAY)
        while True:
            try:
                pages = self.browser.find_elements_by_class_name('clickPage1')
                first_page = int(pages[0].text)
                index = self.current_page - first_page
                current_page = pages[index]
                style = current_page.get_attribute('style')
                if style != u'':
                    print 'Page %d loaded!' % (self.current_page,)
                    break
                else:
                    time.sleep(SLEEP)
            except Exception as e:
                continue

    def save(self):
        database_objects.ExceptionalEnterprise.save(self.ee_list)
        database_objects.ExceptionalReason.save(self.reason_list)

        self.ee_list = []
        self.reason_list = []  # empty the memory

    def accept_alert(self):
        try:
            alert = self.browser.switch_to.alert
            alert.accept()
        except NoAlertPresentException as nape:
            print nape

    def run(self):
        self.get_page_ready()
        while True:
            self.get_one_page()
            print 'This page contains %d items.' % (len(self.ee_list),)
            for i in xrange(len(self.ee_list)):
                index = i + 1
                print 'Getting Item %d...' % (index,)
                try:
                    self.browser.find_element_by_xpath('//*[@id="tbody_id1"]/div[%d]/li[1]/a' % (index,)).send_keys('\n')
                except NoSuchElementException as nsee:
                    print 'Page %d finished!' % (self.current_page,)
                    break
                except ElementNotVisibleException as enve:
                    print 'Discard corresponding reasons! Index: %d.' % (index,)
                    continue
                except UnexpectedAlertPresentException as uape:
                    self.accept_alert()

                self.wait_for_click()

                self.get_reasons()
                self.close_current_tab()

            print 'Dumping into database...'
            self.save()
            print 'Dumping successfully!'
            self.go_next_page()


if __name__ == '__main__':

    my_spider = EESpider(5)

    my_spider.get_page_ready()

    my_spider.run()








